# -*- coding: UTF-8 -*-
import random
import numpy as np
import math
from queue import PriorityQueue
import sys
import config
sys.path.append("..")
from object.lfuTang import LFU
from object.cloudTang import Cloud
import config
from object import imageTang, layerTang, node, task, user, traceTANG, funcs_n2, delay

trajectory = []


class Env:
    def __init__(self):
        self.init_env()

    def seed(self, a):
        self.curseed = config.seed

    def init_env(self):
        self.time = 0
        self.curseed = config.seed

        self.max_cpu = 0
        self.max_mem = 0
        self.max_size = 0
        self.max_layer_size = 0

        self.User = []
        self.mob = []
        self.all_user_task_list = []
        self.layer_lst_on_user = []

        self.Edge = []
        self.user_group = None
        self.edge_task_queue = None

        self.Image = []
        self.Layer = []

        self.Task = []
        self.task_list = []
        self.task_pending = PriorityQueue()

        self.allexist_task_list = []
        self.tp_edgcount = np.zeros((config.EDGE_NODE_NUM + 1))

        self.tsk_operation_time = {}
        self.tsk_mig_dlytime = {}
        self.tsk_backhaul_delaytime = {}
        self.usr_up_trans_dlytime1 = {}
        self.tsk_dwnlod_dlytime = {}
        self.tsk_pending_dlytime = {}
        self.tsk_dwnlod_dlysize = {}
        self.tsk_mig_layer_dlytime = {}
        self.tsk_mig_layer_pendingtime = {}
        self.tsk_mig_layer_downloadsize = {}
        self.tsk_comp_time = {}
        self.tsk_computation_delay = {}
        self.tsk_communication_delay = {}
        self.Migration_delay = {}
        self.down_size = {}
        self.tsk_scaling_delay = {}
        self.tsk_arrange_delay = {}
        self.tsk_waiting_delay = {}

        self.layer_number = 0
        self.image_number = 0
        self.task_number = 0

        self.lfu_cache_layer = {}
        self.lfu_cache_image = {}
        self.cloud = None
        self.task_queue = PriorityQueue()
        self.usr_tsk_starttime = {}
        self.pasta = {}
        self.layer_dict = traceTANG.Trace().layer_stats
        self.image_dict = traceTANG.Trace().image_stats

    def reset(self, task_number=config.TASK_NUM):
        random.seed(self.curseed)
        self.init_env()

        temp_imagelist = list(traceTANG.Trace().image_stats.items())  # 一共有155个image
        temp_imagenum = 155
        temp_imagesize = list(
            random.sample(range(int(0.05 * 1024), 1024 * 5), temp_imagenum)
        )  # 任务大小范围5kb～5Mb
        for index, item in enumerate(temp_imagelist):
            r_image_id = item[1][4]  # requested
            r_image_name = item[0]  # type:str
            r_image_size = [temp_imagesize[index]]
            r_image_feature = item[1][0:3] + r_image_size + [item[1][5]]
            ini_image = imageTang.Image(r_image_id, r_image_name, r_image_feature)
            self.Image.append(ini_image)
        self.image_number = len(self.Image)

        temp_layerlist = list(traceTANG.Trace().layer_stats.items())
        maxsize = 0
        for item in temp_layerlist:
            r_layer_id = item[1][-2]
            r_layer_name = item[0]
            r_layer_feature = item[1][:-2] + [item[1][-1]]
            ini_layer = layerTang.Layer(r_layer_id, r_layer_name, r_layer_feature)
            if ini_layer.size > maxsize:
                maxsize = ini_layer.size
            self.Layer.append(ini_layer)
        self.max_layer_size = maxsize
        self.layer_number = len(self.Layer)

        temp_tasklist = traceTANG.Trace().generate_trace2(
            task_number=task_number,
            arrival_time=0,
            max_num_image=1,
            task_cpu_size_max=50,  # 最多8核 3.5*8
            task_mem_size_max=50,
            task_average_interval=1,
            distribution=config.task_generator_method,
            zipf_param=1.2,
        )
        for item in temp_tasklist:
            r_task_id = item[0]
            start_time = item[1]
            if(start_time==1):
                print("start_time",r_task_id)
            requested_image_name = item[2][0]
            r_task_feature = item[1:]
            requested_image_id = self.image_dict[requested_image_name][-2]
            ini_task = task.Task(
                r_task_id, requested_image_id, r_task_feature, start_time
            )
            self.Task.append(ini_task)
            self.task_list.append(ini_task)
        self.task_number = len(self.Task)

        # initinal Edge
        self.Edge = node.init_node_list_hetero()
        for item in self.Edge:
            item.layer_01_list = np.zeros(self.layer_number)
            for one_lyid in item.ini_lyerid:
                item.layer_01_list[one_lyid] = -1
            item.container_running_list = np.zeros(self.image_number)
            item.image_01_list = np.zeros(self.image_number)

        self.edge_task_queue = np.zeros(
            (config.EDGE_NODE_NUM, self.task_number)
        )  # detailed value depends on tsk's

        task_ids = [i.task_id for i in self.Task]
        tmp_tsk_list = list(np.random.permutation([i.task_id for i in self.Task]))
        # print("是否包含所有原始ID:", set(task_ids) == set(tmp_tsk_list))
        # print("列表长度是否一致:", len(task_ids) == len(tmp_tsk_list))
        new_user = user.GenerateUser()
        taxi_mob_all = new_user.ini_loc(self.curseed)
        tmpUser = []
        for id in range(config.USER_NUM):
            one_mob = taxi_mob_all.get(id)
            user_list = [id, one_mob[0][0], one_mob[0][1]]
            user_mob = [id, one_mob]
            tmpUser.append(user_list)
            self.mob.append(user_mob)

            tmp_sample_tsk_id = tmp_tsk_list.pop()
            self.all_user_task_list.append(tmp_sample_tsk_id)
            self.layer_lst_on_user.append(
                self.generate_user_layer_list2(tmp_sample_tsk_id)
            )

        for id, item in enumerate(tmpUser):
            r_uid = item[0]
            r_user_locationx = item[1]
            r_user_locationy = item[2]
            r_usr_has_tsk = [self.all_user_task_list[id]]
            r_usr_has_lay = self.layer_lst_on_user[id]  #
            r_usr_tsk_arriv_tim_strt = self.Task[
                self.all_user_task_list[id]
            ].task_arrive_time
            r_uid_replace = item[0]
            ini_usr = user.User(
                r_uid,
                r_user_locationx,
                r_user_locationy,
                r_usr_has_tsk,
                r_usr_has_lay,
                r_usr_tsk_arriv_tim_strt,
                r_uid_replace,
            )
            self.User.append(ini_usr)

        for item in self.User:
            self.task_queue.put(
                (item.tsk_arrive_time_start, item.usr_has_tsk[0], item.uid, 0)
            )
            self.allexist_task_list.append(
                (item.tsk_arrive_time_start, item.usr_has_tsk[0], item.uid)
            )

        self.user_group = [[] for i in range(config.EDGE_NODE_NUM + 1)]

        # 6. create cloud
        self.cloud = Cloud(config.cloud_cpu_frequency, config.node_max_bandwidth)

        # 9. init the lfu cache
        self.lfu_cache_layer = {x: LFU() for x in range(config.EDGE_NODE_NUM)}

        tmp_allexist_task_arrivetime = [i for i, j, k in self.allexist_task_list]
        next_uid = [
            k
            for i, j, k in self.allexist_task_list
            if i == min(tmp_allexist_task_arrivetime)
        ][0]

        self.task_current = self.task_list.pop(0)
        self.next_uid_idx = next_uid
        self.next_time = min(tmp_allexist_task_arrivetime)

        self.max_cpu = float(max([i.task_cpu for i in self.Task]))
        self.max_mem = float(max([i.task_mem for i in self.Task]))
        # print(self.max_cpu)
        self.max_size = float(max([i.size for i in self.Image]))
        task_user_map = self.get_task_user_mapping()
        # print(task_user_map)

        obs = self.get_observation()
        done = False
        print("task number: ", str(self.task_number))

        return obs, done

    def get_task_user_mapping(self):
        """
        创建任务到用户的映射
        """
        task_user_map = {}
        for user in self.User:
            for task_id in user.usr_has_tsk:
                task_user_map[task_id] = user.uid
                # if(task_id==0):
                #     print("task_id",task_id,user.uid)
        return task_user_map

    def get_task_layer_request(self, task_id):
        task_image_name = self.Task[task_id].task_image_name[0]

        requested_image_id = [i.id for i in self.Image if i.name == task_image_name][0]

        layer_request_status = [0] * config.Layer_NUM
        layer_sizes = [0] * config.Layer_NUM

        # 遍历所有层，检查是否属于该镜像
        for layer_id, layer in enumerate(self.Layer):
            # 检查层是否属于请求的镜像
            matched_images = [
                i
                for i in self.Image
                if i.name in layer.image_list and i.id == requested_image_id
            ]

            if matched_images:
                layer_request_status[layer_id] = 1
                layer_sizes[layer_id] = layer.size

        return layer_request_status, layer_sizes

    def get_observation(self):
        """
        用户数量: U
        节点数量: N
        层数量: L

        1. 用户信息: U*3
        (1)id (2)x坐标 (3)y坐标

        2. 节点信息: N*14
        (1)可用CPU (2)CPU利用率 (3)可用Mem (4)Mem利用率 (5)磁盘
        (6)磁盘利用率 (7)容器数量 (8)x (9)y (10)用户带宽 (11)无线带宽
        (12)任务数量 (13)等待时间 (14)迁移次数

        3. 节点容器: N*(2L)
        (1)层 (2)层下载时间

        4. 任务信息：4
        (1)当前请求CPU (2)当前请求Mem (3)当前开始时间 (4)当前所属用户

        5. 任务容器：L*2
        (1)任务所需的层 (2)任务所需层的大小

        """
        # 1. 用户信息处理 ##############################
        user_info = []
        for user in self.User:
            user_info.append(
                [
                    user.uid,  # 用户ID
                    user.user_locationx,  # x坐标
                    user.user_locationy,  # y坐标
                ]
            )
        user_info = np.array(user_info)

        # 2. 边缘节点信息处理 ##############################
        nodes_info = []

        for node in self.Edge:
            node_resources = [
                node.available_cpu,  # 可用CPU
                1 - node.available_cpu / node.cpu,  # CPU利用率
                node.available_mem,  # 可用内存
                1 - node.available_mem / node.mem,  # 内存利用率
                node.available_disk,  # 可用磁盘
                1 - node.available_disk / node.disk,  # 磁盘利用率
                node.container_number,  # 容器数量
                node.location_x,  # x坐标
                node.location_y,  # y坐标
                node.node_bw,  # 用户带宽
                node.node_bw2,  # 无线带宽
            ]

            # 节点任务信息
            node_tasks = [task for task in self.Task if task.assigned_node == node.id]
            node_task_info = [
                len(node_tasks),  # 任务数量
                (
                    sum(self.time - task.wait_time for task in node_tasks)
                    if node_tasks
                    else 0
                ),  # 总等待时间
                sum(task.reschedule_count for task in node_tasks),  # 迁移次数
            ]

            nodes_info.append(node_resources + node_task_info)
        nodes_info = np.array(nodes_info)

        # 节点层信息处理
        nodes_layers = []
        for node in self.Edge:
            layer_states = []
            layer_download_times = []
            for layer_id, layer_state in enumerate(node.layer_01_list):
                if layer_state == 0:  # 层未下载
                    download_time = self.Layer[layer_id].size / (node.node_bw2 / 8)
                    max_ongoing_download_time = max(
                        [
                            layer_finish_time - self.time
                            for layer_finish_time in node.layer_01_list
                            if layer_finish_time > self.time
                        ],
                        default=0,
                    )
                    estimated_finish_time = (
                        self.time + max_ongoing_download_time + download_time
                    )
                    layer_download_times.append(estimated_finish_time - self.time)
                    layer_states.append(layer_id)

                elif layer_state == -1:  # 层已下载
                    layer_download_times.append(0)
                    layer_states.append(layer_id)

                elif layer_state > self.time:  # 层正在下载
                    layer_download_times.append(layer_state - self.time)
                    layer_states.append(layer_id)

                else:
                    layer_download_times.append(0)
                    layer_states.append(layer_id)

            nodes_layers.append([layer_states, layer_download_times])
        nodes_layers = np.array(nodes_layers)
 
        # 3. 当前任务处理 ##############################  
        tasks_info = [0, 0, 0, 0]  
        tasks_layers = np.zeros((2, config.Layer_NUM), dtype=float)  # 使用固定大小的全零数组  
        task_user_map = self.get_task_user_mapping()  

        if self.allexist_task_list:  
            # 获取当前任务  
            current_task_idx = [  
                item.usr_has_tsk for item in self.User if item.uid == self.next_uid_idx  
            ][0][0]  

            # 当前任务信息  
            tasks_info = [  
                # self.Task[current_task_idx].task_cpu / self.max_cpu,  
                # self.Task[current_task_idx].task_mem / self.max_mem,  
                self.Task[current_task_idx].task_cpu ,  
                self.Task[current_task_idx].task_mem , 
                self.Task[current_task_idx].start_time,  
                task_user_map.get(self.Task[current_task_idx].task_id, -1)  
            ]  

            # 当前任务层信息  
            layer_request_status, layer_sizes = self.get_task_layer_request(  
                self.Task[current_task_idx].task_id  
            )  
            
            # 确保 tasks_layers 总是固定大小  
            tasks_layers = np.zeros((2, config.Layer_NUM), dtype=float)  
            tasks_layers[0, :len(layer_request_status)] = layer_request_status  
            tasks_layers[1, :len(layer_sizes)] = layer_sizes  

        # 确保 tasks_info 总是长度为 4  
        tasks_info = tasks_info[:4] + [0] * (4 - len(tasks_info))  

        # 4. 组装最终观察状态 ##############################
        obs = {
            "users": user_info,
            "nodes": nodes_info,
            "nodes_layers": nodes_layers,
            "tasks": tasks_info,
            "tasks_layers": tasks_layers,
        }
        # print("obs[users]",np.shape(obs["users"]))
        # print("obs[nodes]",np.shape(obs["nodes"]))
        # print("obs[nodes_layers]",np.shape(obs["nodes_layers"]))
        # print("obs[tasks]",np.shape(obs["tasks"]))
        # print("obs[tasks_layers]",np.shape(obs["tasks_layers"]))
        # print("obs[users]",obs["users"])
        # print("obs[nodes]",obs["nodes"])
        # print("obs[nodes_layers]",obs["nodes_layers"])
        # print("obs[tasks]",obs["tasks"])
        # print("obs[tasks_layers]",obs["tasks_layers"])

        return obs

    def is_done(self):
        if self.task_queue.empty() == False or self.task_pending.empty() == False:
            # print(self.task_queue.qsize())
            # print(self.task_pending.qsize())
            done = False
        else:
            print("the env is end. ", "……" * 20)
            done = True
        return done

    def step(self, action):
        # 问题：这个是指迁移的顺序吗
        mig_order = 1
        reward = 0
        count_num_on_cloud = 0
        # 下一个任务没有到达时，处理运行队列中的任务
        while self.time != self.next_time:
            tmp_pendtsk_list = []
            while self.task_pending.empty() == False:
                task_pending_tmp = self.task_pending.get()
                tmp_pendtsk_list.append(task_pending_tmp)
                # print("task_pending_tmp is: ", task_pending_tmp)
                node_id = self.Task[task_pending_tmp[2]].assigned_node
                if task_pending_tmp[0] + task_pending_tmp[1] > self.time:
                    # 这里检查任务是否超过通讯距离，从而进行被动伸缩
                    if self.check_d(task_pending_tmp):
                        self.task_queue.put(
                            (
                                self.next_time + mig_order,
                                task_pending_tmp[2],
                                task_pending_tmp[3],
                                task_pending_tmp[-1] + 1,
                            )
                        )
                        # waiting time
                        task_temp = self.Task[task_pending_tmp[2]]
                        task_temp.wait_time = self.time
                        task_temp.reschedule_count += 1

                        self.allexist_task_list.append(
                            (
                                self.next_time + mig_order,
                                task_pending_tmp[2],
                                task_pending_tmp[3],
                            )
                        )
                        mig_order += 1
                        tmp_pendtsk_list.remove(task_pending_tmp)
                        # print("task id is："+str(task_pending_tmp[2])+" passive scaling 1 **************************************************")

                    elif node_id:
                        if node_id != config.EDGE_NODE_NUM:
                            node = self.Edge[node_id]
                            if (
                                node.available_mem < config.LIMITED_MEMORY * node.mem
                                or node.available_cpu < config.LIMITED_CPU * node.cpu
                            ):
                                self.task_queue.put(
                                    (
                                        self.next_time + mig_order,
                                        task_pending_tmp[2],
                                        task_pending_tmp[3],
                                        task_pending_tmp[-1] + 1,
                                    )
                                )

                                # waiting time
                                task_temp = self.Task[task_pending_tmp[2]]
                                task_temp.wait_time = self.time
                                task_temp.reschedule_count += 1

                                self.allexist_task_list.append(
                                    (
                                        self.next_time + mig_order,
                                        task_pending_tmp[2],
                                        task_pending_tmp[3],
                                    )
                                )

                                mig_order += 1
                                tmp_pendtsk_list.remove(task_pending_tmp)

                                # print("task id is："+str(task_pending_tmp[2])+" active scaling 1 **************************************************")

                    else:
                        continue
                else:
                    continue

            if tmp_pendtsk_list:
                for item in tmp_pendtsk_list:
                    self.task_pending.put(item)
            self.time += 1

            for idx, usr in enumerate(self.User):
                self.one_user_loc_update(idx, self.time)

        # start
        current_time = self.time

        # 2 time_arrived to get current task == pending_task_tuple
        if self.task_queue.empty() == False:
            pending_task_tuple = self.task_queue.get()
            # print(pending_task_tuple[0])
            # print(current_time)
            # if pending_task_tuple[0] != current_time:  
            #     print(pending_task_tuple[0])
            #     print(current_time)
                # self.task_queue.put(pending_task_tuple)  
                # return obs, 0, None, done, []  
            # 获取当前任务的各种信息
            if pending_task_tuple[0] == current_time:
                pending_tskid = pending_task_tuple[1]
                cur_uid = pending_task_tuple[2]

                usr_belong_edgeid = action

                # 获取task所在的node id
                tmp_p = self.thruUsrGroup_find_usrCurEdg(pending_tskid)

                if tmp_p is not None:
                    usr_previous_edgeid = tmp_p
                else:
                    usr_previous_edgeid = action

                curnt_task_siz = [
                    i.size
                    for i in self.Image
                    if i.name == self.Task[pending_tskid].task_image_name[0]
                ][0]
                requested_imge_id = [
                    i.id
                    for i in self.Image
                    if i.name == self.Task[pending_tskid].task_image_name[0]
                ][0]
                current_user_has_layers = [
                    subuser.usr_has_lay
                    for ind, subuser in enumerate(self.User)
                    if subuser.uid == cur_uid
                ][0]
            else:
                raise EnvironmentError

            # 3 update lfu_cache_layer
            if usr_belong_edgeid != config.EDGE_NODE_NUM:
                for layer_id in self.Edge[usr_belong_edgeid].ini_lyerid:
                    self.lfu_cache_layer[int(usr_belong_edgeid)].put(layer_id)
                self.tp_edgcount[action] += 1

            if usr_belong_edgeid == config.EDGE_NODE_NUM:
                # print("usr_belong_edgeid is : ", str(usr_belong_edgeid))
                count_num_on_cloud += 1
            # print("usr_previous_edgeid is: ", str(usr_previous_edgeid))

            # 4 cloud
            if usr_belong_edgeid == config.EDGE_NODE_NUM:
                if usr_previous_edgeid == usr_belong_edgeid:

                    self.usr_up_trans_dlytime1.update({(pending_tskid, cur_uid): 0.05})
                    self.tsk_backhaul_delaytime.update({(pending_tskid, cur_uid): 0.05})
                    self.tsk_mig_dlytime.update({(pending_tskid, cur_uid): 0})
                    self.tsk_waiting_delay.update(({(pending_tskid, cur_uid): 0}))

                    dwnlod_siz = 0
                    for one in [
                        subuser.usr_has_lay
                        for ind, subuser in enumerate(self.User)
                        if subuser.uid == cur_uid
                    ][0]:
                        dwnlod_siz += self.Layer[one].size

                    tmp_download_time = dwnlod_siz / (self.cloud.bandwidth / 8)

                    self.down_size.update({(pending_tskid, cur_uid): dwnlod_siz})
                    self.tsk_dwnlod_dlytime.update(
                        {(pending_tskid, cur_uid): tmp_download_time}
                    )

                    tmp_computation_time = (
                        self.Task[pending_tskid].task_cpu / self.cloud.cpu_frequency
                    )

                    self.tsk_computation_delay.update(
                        {(pending_tskid, cur_uid): tmp_computation_time}
                    )
                    self.tsk_mig_layer_dlytime.update({(pending_tskid, cur_uid): 0})
                    self.tsk_mig_layer_downloadsize.update(
                        {(pending_tskid, cur_uid): 0}
                    )
                    self.tsk_mig_layer_pendingtime.update(
                        {(pending_tskid, cur_uid): tmp_download_time}
                    )

                    self.Task[pending_tskid].assigned_node = config.EDGE_NODE_NUM

                else:
                    self.usr_up_trans_dlytime1.update({(pending_tskid, cur_uid): 0})
                    self.tsk_backhaul_delaytime.update({(pending_tskid, cur_uid): 0.05})
                    self.tsk_mig_dlytime.update({(pending_tskid, cur_uid): 0.05})

                    waiting_time = current_time - self.Task[pending_tskid].wait_time
                    self.tsk_waiting_delay.update(
                        ({(pending_tskid, cur_uid): waiting_time})
                    )

                    dwnlod_siz = 0
                    for one in [
                        subuser.usr_has_lay
                        for ind, subuser in enumerate(self.User)
                        if subuser.uid == cur_uid
                    ][0]:
                        dwnlod_siz += self.Layer[one].size

                    tmp_download_time = dwnlod_siz / (self.cloud.bandwidth / 8)

                    self.down_size.update({(pending_tskid, cur_uid): 0})
                    self.tsk_dwnlod_dlytime.update({(pending_tskid, cur_uid): 0})

                    tmp_computation_time = (
                        self.Task[pending_tskid].task_cpu / self.cloud.cpu_frequency
                    )
                    self.tsk_computation_delay.update(
                        {(pending_tskid, cur_uid): tmp_computation_time}
                    )

                    self.tsk_mig_layer_dlytime.update(
                        {(pending_tskid, cur_uid): tmp_download_time}
                    )
                    self.tsk_mig_layer_downloadsize.update(
                        {(pending_tskid, cur_uid): dwnlod_siz}
                    )
                    self.tsk_mig_layer_pendingtime.update(
                        {(pending_tskid, cur_uid): tmp_download_time}
                    )

                    self.Task[pending_tskid].assigned_node = config.EDGE_NODE_NUM

                    # release the resource of previous edge
                    self.Edge[usr_previous_edgeid].available_cpu += self.Task[
                        pending_tskid
                    ].task_cpu
                    self.Edge[usr_previous_edgeid].available_mem += self.Task[
                        pending_tskid
                    ].task_mem
                    self.remove_layers(usr_previous_edgeid, pending_tskid)
                    self.Edge[usr_previous_edgeid].container_running_list[
                        requested_imge_id
                    ] -= 1
                    self.Edge[usr_previous_edgeid].image_01_list[requested_imge_id] = 0
                    self.Edge[usr_previous_edgeid].container_number -= 1
                    self.edge_task_queue[usr_previous_edgeid][pending_tskid] = 0
                    # print("迁移到云上，释放节点上的资源")

                scaling_delay = max(
                    self.tsk_mig_dlytime.get((pending_tskid, cur_uid), None),
                    self.tsk_mig_layer_dlytime.get((pending_tskid, cur_uid), None),
                ) + config.beta * self.tsk_waiting_delay.get(
                    (pending_tskid, cur_uid), None
                )

                self.tsk_scaling_delay.update({(pending_tskid, cur_uid): scaling_delay})

                computation_delay = self.tsk_computation_delay.get(
                    (pending_tskid, cur_uid), None
                )

                arrange_delay = max(
                    self.usr_up_trans_dlytime1.get((pending_tskid, cur_uid), None),
                    (self.tsk_dwnlod_dlytime.get((pending_tskid, cur_uid), None)) * 0.4,
                )
                # print("usr_up_trans_dlytime1:"+str(self.usr_up_trans_dlytime1.get((pending_tskid, cur_uid))*1000))
                # print("tsk_dwnlod_dlytime:"+str(self.tsk_dwnlod_dlytime.get((pending_tskid, cur_uid))*1000))
                self.tsk_arrange_delay.update({(pending_tskid, cur_uid): arrange_delay})

                backhaul_delay = self.tsk_backhaul_delaytime.get(
                    (pending_tskid, cur_uid), None
                )

                communication_delay = backhaul_delay + self.usr_up_trans_dlytime1.get(
                    (pending_tskid, cur_uid), None
                )
                self.tsk_communication_delay.update(
                    {(pending_tskid, cur_uid): communication_delay}
                )

                tsk_total_operationTime = (
                    arrange_delay + computation_delay + scaling_delay + backhaul_delay
                )

                # tsk_total_operationTime_1 = scaling_delay + computation_delay + backhaul_delay

                self.tsk_operation_time.update(
                    {(pending_tskid, cur_uid): tsk_total_operationTime}
                )

                self.allexist_task_list.remove((current_time, pending_tskid, cur_uid))

                self.task_pending.put(
                    (
                        tsk_total_operationTime,
                        pending_task_tuple[0],
                        pending_tskid,
                        cur_uid,
                        usr_belong_edgeid,
                        pending_task_tuple[-1],
                    )
                )

                reward = tsk_total_operationTime

            else:
                # 5. calculate the tsk mig to target node (usr-edgeid in action != usr-edgeid in self.usr-group)
                # 发生迁移
                if usr_belong_edgeid != usr_previous_edgeid:
                    # 从节点迁移，而不是云上
                    if usr_previous_edgeid != config.EDGE_NODE_NUM:

                        waiting_time = current_time - self.Task[pending_tskid].wait_time
                        self.tsk_waiting_delay.update(
                            ({(pending_tskid, cur_uid): waiting_time})
                        )

                        previous_latx = self.Edge[usr_previous_edgeid].location_x
                        previous_lony = self.Edge[usr_previous_edgeid].location_y
                        belong_latx = self.Edge[usr_belong_edgeid].location_x
                        belong_lony = self.Edge[usr_belong_edgeid].location_y
                        # 迁移的距离
                        mig_d = delay.hop_distance(
                            previous_latx, previous_lony, belong_latx, belong_lony
                        )

                        uid_current_x = [
                            itm.user_locationx
                            for ind, itm in enumerate(self.User)
                            if itm.uid == pending_task_tuple[2]
                        ][0]
                        uid_current_y = [
                            itm.user_locationy
                            for ind, itm in enumerate(self.User)
                            if itm.uid == pending_task_tuple[2]
                        ][0]
                        # 通讯回程的距离
                        backhual_d = delay.hop_distance(
                            uid_current_x, uid_current_y, belong_latx, belong_lony
                        )

                        # calculate the migration delay
                        # 问题：从节点上迁移的回程距离是两个节点之间的距离
                        # random.uniform(0.5,100) [0.5, 100] MB
                        task_mig_delay = (
                            random.uniform(0.5, 100)
                            * 8
                            / min(
                                self.Edge[usr_belong_edgeid].node_bw,
                                self.Edge[usr_previous_edgeid].node_bw,
                            )
                            + random.uniform(1, 3) * mig_d
                        )

                        # calculate the backhual delay
                        # random.uniform(0.05,5) [0.05, 5] MB  node_bw2 Mbps,
                        task_backhaul_delay = (
                            random.uniform(0.05, 5)
                            * 8
                            / self.Edge[usr_belong_edgeid].node_bw2
                            + 0.02 * backhual_d
                        )

                        download_siz = 0
                        download_tmp = 0
                        pending_download_time = 0
                        download_finish_time = current_time

                        for item in current_user_has_layers:
                            get_layer_state_on_edg = self.Edge[
                                usr_belong_edgeid
                            ].layer_01_list[item]
                            if get_layer_state_on_edg == 0:
                                one_download_size = self.Layer[item].size
                                download_siz += one_download_size
                                # 问题：节点上下载，应该是有线的还是无线的
                                one_download_time = one_download_size / (
                                    self.Edge[usr_belong_edgeid].node_bw2 / 8
                                )
                                download_tmp += one_download_time

                                download_finish_time = current_time + download_tmp
                                self.Edge[usr_belong_edgeid].download_finish_time = (
                                    download_finish_time
                                )
                                self.Edge[usr_belong_edgeid].layer_01_list[
                                    item
                                ] = download_finish_time
                                # 问题：等待下载时间有问题，我觉得可以给每一个Task加一个等待起始时间，默认为0
                                if (
                                    self.Edge[usr_belong_edgeid].download_finish_time
                                    - current_time
                                    > pending_download_time
                                ):
                                    pending_download_time = (
                                        self.Edge[
                                            usr_belong_edgeid
                                        ].download_finish_time
                                        - current_time
                                    )

                            elif get_layer_state_on_edg == -1:

                                pass

                            else:
                                # 小于当前时间，说明layer下载完成
                                if get_layer_state_on_edg < current_time:
                                    self.Edge[usr_belong_edgeid].layer_01_list[
                                        item
                                    ] = -1
                                    self.Edge[
                                        usr_belong_edgeid
                                    ].available_disk -= self.Layer[item].size
                                    self.Edge[usr_belong_edgeid].ini_lyerid += [item]

                        # 检查每个node中layer的下载情况
                        for idx in range(config.EDGE_NODE_NUM):
                            for id, one_state in enumerate(
                                self.Edge[idx].layer_01_list
                            ):
                                if one_state > 0 and one_state < current_time:
                                    self.Edge[idx].layer_01_list[id] = -1
                                    self.Edge[idx].available_disk -= self.Layer[id].size
                                    self.Edge[idx].ini_lyerid += [id]

                        self.down_size.update({(pending_tskid, cur_uid): 0})
                        self.tsk_mig_dlytime.update(
                            {(pending_tskid, cur_uid): task_mig_delay}
                        )
                        self.tsk_mig_layer_dlytime.update(
                            {(pending_tskid, cur_uid): download_tmp}
                        )
                        self.usr_up_trans_dlytime1.update({(pending_tskid, cur_uid): 0})
                        self.tsk_dwnlod_dlytime.update({(pending_tskid, cur_uid): 0})
                        self.tsk_backhaul_delaytime.update(
                            {(pending_tskid, cur_uid): task_backhaul_delay}
                        )

                        self.Task[pending_tskid].assigned_node = usr_belong_edgeid

                        self.tsk_mig_layer_pendingtime.update(
                            {(pending_tskid, cur_uid): pending_download_time}
                        )
                        self.tsk_mig_layer_downloadsize.update(
                            {(pending_tskid, cur_uid): download_siz}
                        )

                        # 5.1 update other resources for the other delay (cpu, mem)
                        self.Edge[usr_previous_edgeid].available_cpu += self.Task[
                            pending_tskid
                        ].task_cpu
                        self.Edge[usr_previous_edgeid].available_mem += self.Task[
                            pending_tskid
                        ].task_mem
                        self.remove_layers(usr_previous_edgeid, pending_tskid)
                        self.Edge[usr_previous_edgeid].container_running_list[
                            requested_imge_id
                        ] -= 1
                        self.Edge[usr_previous_edgeid].image_01_list[
                            requested_imge_id
                        ] = 0
                        self.Edge[usr_previous_edgeid].container_number -= 1
                        self.edge_task_queue[usr_previous_edgeid][pending_tskid] = 0
                        # print("从节点上迁移到节点上，释放资源")

                    else:

                        waiting_time = current_time - self.Task[pending_tskid].wait_time
                        self.tsk_waiting_delay.update(
                            ({(pending_tskid, cur_uid): waiting_time})
                        )

                        # 任务部署时间
                        # 如果从云上迁移下来，通讯时间该如何计算
                        task_mig_delay = (
                            random.uniform(0.5, 100)
                            * 8
                            / min(
                                self.Edge[usr_belong_edgeid].node_bw,
                                self.cloud.bandwidth,
                            )
                        )

                        task_backhaul_delay = (
                            random.uniform(0.05, 5)
                            * 8
                            / self.Edge[usr_belong_edgeid].node_bw2
                        )
                        print("migrate from cloud *****************")

                        download_siz = 0
                        download_tmp = 0
                        pending_download_time = 0
                        download_finish_time = current_time

                        for item in current_user_has_layers:
                            get_layer_state_on_edg = self.Edge[
                                usr_belong_edgeid
                            ].layer_01_list[item]
                            if get_layer_state_on_edg == 0:
                                one_download_size = self.Layer[item].size
                                download_siz += one_download_size

                                one_download_time = one_download_size / (
                                    self.Edge[usr_belong_edgeid].node_bw2 / 8
                                )
                                download_tmp += one_download_time

                                download_finish_time = current_time + download_tmp
                                self.Edge[usr_belong_edgeid].download_finish_time = (
                                    download_finish_time
                                )
                                self.Edge[usr_belong_edgeid].layer_01_list[
                                    item
                                ] = download_finish_time
                                # 问题：等待下载时间有问题，我觉得可以给每一个Task加一个等待起始时间，默认为0
                                if (
                                    self.Edge[usr_belong_edgeid].download_finish_time
                                    - current_time
                                    > pending_download_time
                                ):
                                    pending_download_time = (
                                        self.Edge[
                                            usr_belong_edgeid
                                        ].download_finish_time
                                        - current_time
                                    )

                            elif get_layer_state_on_edg == -1:

                                pass

                            else:
                                if get_layer_state_on_edg < current_time:
                                    self.Edge[usr_belong_edgeid].layer_01_list[
                                        item
                                    ] = -1
                                    self.Edge[
                                        usr_belong_edgeid
                                    ].available_disk -= self.Layer[item].size
                                    self.Edge[usr_belong_edgeid].ini_lyerid += [item]

                        for idx in range(config.EDGE_NODE_NUM):
                            for id, one_state in enumerate(
                                self.Edge[idx].layer_01_list
                            ):
                                if one_state > 0 and one_state < current_time:
                                    self.Edge[idx].layer_01_list[id] = -1
                                    self.Edge[idx].available_disk -= self.Layer[id].size
                                    self.Edge[idx].ini_lyerid += [id]

                        self.down_size.update({(pending_tskid, cur_uid): 0})
                        self.tsk_mig_dlytime.update(
                            {(pending_tskid, cur_uid): task_mig_delay}
                        )
                        self.tsk_mig_layer_dlytime.update(
                            {(pending_tskid, cur_uid): download_tmp}
                        )
                        self.usr_up_trans_dlytime1.update({(pending_tskid, cur_uid): 0})
                        self.tsk_dwnlod_dlytime.update({(pending_tskid, cur_uid): 0})
                        self.tsk_backhaul_delaytime.update(
                            {(pending_tskid, cur_uid): task_backhaul_delay}
                        )

                        self.Task[pending_tskid].assigned_node = usr_belong_edgeid

                        self.tsk_mig_layer_pendingtime.update(
                            {(pending_tskid, cur_uid): pending_download_time}
                        )
                        self.tsk_mig_layer_downloadsize.update(
                            {(pending_tskid, cur_uid): download_siz}
                        )
                        # print("从云上迁移到节点上，不需要释放资源")

                # 不发生迁移
                else:
                    self.tsk_waiting_delay.update(({(pending_tskid, cur_uid): 0}))

                    belong_latx = self.Edge[usr_belong_edgeid].location_x
                    belong_lony = self.Edge[usr_belong_edgeid].location_y

                    uid_current_x = [
                        itm.user_locationx
                        for ind, itm in enumerate(self.User)
                        if itm.uid == pending_task_tuple[2]
                    ][0]
                    uid_current_y = [
                        itm.user_locationy
                        for ind, itm in enumerate(self.User)
                        if itm.uid == pending_task_tuple[2]
                    ][0]
                    # 通讯回程的距离
                    backhual_d = delay.hop_distance(
                        uid_current_x, uid_current_y, belong_latx, belong_lony
                    )

                    # calculate the backhual delay
                    task_backhaul_delay = (
                        random.uniform(0.05, 5)
                        * 8
                        / self.Edge[usr_belong_edgeid].node_bw2
                        + 0.02 * backhual_d
                    )

                    # calculate the access delay
                    tmp_usr_up_trans_dlytime1, distance, trate = delay.trans_dly(
                        curnt_task_siz,
                        uid_current_x,
                        uid_current_y,
                        belong_latx,
                        belong_lony,
                        self.Edge[usr_belong_edgeid].node_bw2,
                    )
                    download_siz = 0
                    download_tmp = 0
                    pending_download_time = 0
                    download_finish_time = current_time

                    for item in current_user_has_layers:
                        get_layer_state_on_edg = self.Edge[
                            usr_belong_edgeid
                        ].layer_01_list[item]
                        if get_layer_state_on_edg == 0:
                            one_download_size = self.Layer[item].size
                            download_siz += one_download_size

                            # one_download_size: MB, node_bw2 Mb
                            one_download_time = one_download_size / (
                                self.Edge[usr_belong_edgeid].node_bw2 / 8
                            )
                            download_tmp += one_download_time

                            download_finish_time = current_time + download_tmp
                            self.Edge[usr_belong_edgeid].download_finish_time = (
                                download_finish_time
                            )
                            self.Edge[usr_belong_edgeid].layer_01_list[
                                item
                            ] = download_finish_time

                            if (
                                self.Edge[usr_belong_edgeid].download_finish_time
                                - current_time
                                > pending_download_time
                            ):
                                pending_download_time = (
                                    self.Edge[usr_belong_edgeid].download_finish_time
                                    - current_time
                                )

                        elif get_layer_state_on_edg == -1:

                            pass

                        else:
                            if get_layer_state_on_edg < current_time:
                                self.Edge[usr_belong_edgeid].layer_01_list[item] = -1
                                self.Edge[
                                    usr_belong_edgeid
                                ].available_disk -= self.Layer[item].size
                                self.Edge[usr_belong_edgeid].ini_lyerid += [item]

                    for idx in range(config.EDGE_NODE_NUM):
                        for id, one_state in enumerate(self.Edge[idx].layer_01_list):
                            if one_state > 0 and one_state < current_time:
                                self.Edge[idx].layer_01_list[id] = -1
                                self.Edge[idx].available_disk -= self.Layer[id].size
                                self.Edge[idx].ini_lyerid += [id]

                    self.Task[pending_tskid].assigned_node = usr_belong_edgeid
                    # print("self.Task[pending_tskid].assigned_node ",self.Task[pending_tskid].assigned_node )

                    self.down_size.update({(pending_tskid, cur_uid): download_siz})
                    self.tsk_dwnlod_dlytime.update(
                        {(pending_tskid, cur_uid): download_tmp}
                    )

                    self.tsk_mig_dlytime.update({(pending_tskid, cur_uid): 0})
                    self.tsk_mig_layer_dlytime.update({(pending_tskid, cur_uid): 0})
                    self.tsk_mig_layer_downloadsize.update(
                        {(pending_tskid, cur_uid): 0}
                    )

                    self.tsk_mig_layer_pendingtime.update(
                        {(pending_tskid, cur_uid): pending_download_time}
                    )

                    self.tsk_backhaul_delaytime.update(
                        {(pending_tskid, cur_uid): task_backhaul_delay}
                    )

                    self.usr_up_trans_dlytime1.update(
                        {(pending_tskid, cur_uid): tmp_usr_up_trans_dlytime1}
                    )

                # 6 cal trans
                if True:
                    self.Edge[usr_belong_edgeid].available_cpu -= self.Task[
                        pending_tskid
                    ].task_cpu
                    self.Edge[usr_belong_edgeid].available_mem -= self.Task[
                        pending_tskid
                    ].task_mem
                    self.Edge[usr_belong_edgeid].container_running_list[
                        requested_imge_id
                    ] += 1
                    self.Edge[usr_belong_edgeid].image_01_list[requested_imge_id] = 1
                    self.Edge[usr_belong_edgeid].container_number += 1
                    self.edge_task_queue[usr_belong_edgeid][pending_tskid] = 1
                    # print("任务占用资源")
                else:
                    print(
                        "this task has beed offloaded to edge, it might be through mig-process."
                    )

                # calculate the computation delay
                task_process_density = config.task_process_density
                current_workload = self.workload(usr_belong_edgeid)
                computation_delay = (
                    current_workload
                    + curnt_task_siz * 1024 * 1024 * 8 * task_process_density
                ) / self.Edge[
                    usr_belong_edgeid
                ].cpu_capacity  # Mb*1024*1024 = b    b* 1000 circle/bit = circle
                self.tsk_computation_delay.update(
                    {(pending_tskid, cur_uid): computation_delay}
                )

                scaling_delay = max(
                    self.tsk_mig_dlytime.get((pending_tskid, cur_uid), None),
                    self.tsk_mig_layer_dlytime.get((pending_tskid, cur_uid), None),
                ) + config.beta * self.tsk_waiting_delay.get(
                    (pending_tskid, cur_uid), None
                )

                self.tsk_scaling_delay.update({(pending_tskid, cur_uid): scaling_delay})

                arrange_delay = max(
                    self.usr_up_trans_dlytime1.get((pending_tskid, cur_uid), None),
                    (self.tsk_dwnlod_dlytime.get((pending_tskid, cur_uid), None)) * 0.4,
                )
                # print("usr_up_trans_dlytime1:"+str(self.usr_up_trans_dlytime1.get((pending_tskid, cur_uid))*1000))
                # print("tsk_dwnlod_dlytime:"+str(self.tsk_dwnlod_dlytime.get((pending_tskid, cur_uid))*1000))
                self.tsk_arrange_delay.update({(pending_tskid, cur_uid): arrange_delay})

                backhaul_delay = self.tsk_backhaul_delaytime.get(
                    (pending_tskid, cur_uid), None
                )

                access_delay = self.usr_up_trans_dlytime1.get(
                    (pending_tskid, cur_uid), None
                )
                communication_delay = access_delay + backhaul_delay
                self.tsk_communication_delay.update(
                    {(pending_tskid, cur_uid): communication_delay}
                )

                tsk_total_operationTime = (
                    scaling_delay + arrange_delay + backhaul_delay + computation_delay
                )

                # tsk_total_operationTime_1 = scaling_delay + computation_delay + backhaul_delay
                self.tsk_operation_time.update(
                    {(pending_tskid, cur_uid): tsk_total_operationTime}
                )

                # 6.3 update each reward
                reward = tsk_total_operationTime

                self.allexist_task_list.remove((current_time, pending_tskid, cur_uid))
                self.task_pending.put(
                    (
                        tsk_total_operationTime,
                        pending_task_tuple[0],
                        pending_tskid,
                        cur_uid,
                        usr_belong_edgeid,
                        pending_task_tuple[-1],
                    )
                )

            # 7. update usr_lst
            for usr_lst in self.user_group:
                if cur_uid in usr_lst:
                    usr_lst.remove(cur_uid)
            self.user_group[action] += [cur_uid]

        # 8. delete already done
        tmp_p = []
        while self.task_pending.empty() == False:
            task_pending = self.task_pending.get()
            # print("task_pending_tmp1 is: ", task_pending)
            tmp_p.append(task_pending)
            task_pending_operatime = task_pending[0]

            if task_pending_operatime + task_pending[1] < self.time:
                self.end_pending_task(task_pending[2], task_pending[3], task_pending[4])
                tmp_p.remove(task_pending)
        if tmp_p:
            for item in tmp_p:
                self.task_pending.put(item)

        # 9. get next_uid
        if self.allexist_task_list:
            next_task_id = []
            tmp_allexist_task_arrivetime2 = [i for i, j, k in self.allexist_task_list]
            next_uid = [
                k
                for i, j, k in self.allexist_task_list
                if i == min(tmp_allexist_task_arrivetime2)
            ]
            self.next_uid_idx = next_uid[0]
            min_task_id = [
                item.usr_has_tsk for item in self.User if item.uid == next_uid[0]
            ][0][0]
            for id in next_uid:
                task_id = [item.usr_has_tsk for item in self.User if item.uid == id][0][
                    0
                ]
                if task_id < min_task_id:
                    min_task_id = task_id
                    self.next_uid_idx = id
                next_task_id.append(task_id)

            self.next_time = min(tmp_allexist_task_arrivetime2)
            # check constraints and ends tasks

        elif self.task_queue.empty() == True or self.task_pending.empty() == False:
            break2 = False

            while self.task_queue.empty() == True:
                # check wether task done
                tmp_pendtsk_list2 = []
                while self.task_pending.empty() == False:
                    task_pending_tmp = self.task_pending.get()
                    # print("task_pending_tmp2 is: ",task_pending_tmp)
                    tmp_pendtsk_list2.append(task_pending_tmp)
                    node_id = self.Task[task_pending_tmp[2]].assigned_node

                    if task_pending_tmp[0] + task_pending_tmp[1] > self.time:
                        if self.check_d(task_pending_tmp):
                            self.task_queue.put(
                                (
                                    self.time + mig_order,
                                    task_pending_tmp[2],
                                    task_pending_tmp[3],
                                    task_pending_tmp[-1] + 1,
                                )
                            )
                            self.allexist_task_list.append(
                                (
                                    self.time + mig_order,
                                    task_pending_tmp[2],
                                    task_pending_tmp[3],
                                )
                            )

                            task_temp = self.Task[task_pending_tmp[2]]
                            task_temp.wait_time = self.time
                            task_temp.reschedule_count += 1

                            mig_order += 1
                            tmp_pendtsk_list2.remove(task_pending_tmp)
                            break2 = True
                            print(
                                "task id is："
                                + str(task_pending_tmp[2])
                                + " passive scaling 2 **************************************************"
                            )

                            break
                        elif node_id:
                            if node_id < config.EDGE_NODE_NUM:
                                node = self.Edge[node_id]
                                if (
                                    node.available_mem
                                    < config.LIMITED_MEMORY * node.mem
                                    or node.available_cpu
                                    < config.LIMITED_CPU * node.cpu
                                ):
                                    self.task_queue.put(
                                        (
                                            self.time + mig_order,
                                            task_pending_tmp[2],
                                            task_pending_tmp[3],
                                            task_pending_tmp[-1] + 1,
                                        )
                                    )
                                    # waiting time
                                    task_temp = self.Task[task_pending_tmp[2]]
                                    task_temp.wait_time = self.time
                                    task_temp.reschedule_count += 1

                                    self.allexist_task_list.append(
                                        (
                                            self.time + mig_order,
                                            task_pending_tmp[2],
                                            task_pending_tmp[3],
                                        )
                                    )
                                    mig_order += 1
                                    tmp_pendtsk_list2.remove(task_pending_tmp)
                                    break2 = True
                                    # print(
                                    #     "task id is："
                                    #     + str(task_pending_tmp[2])
                                    #     + " active scaling 2 **************************************************"
                                    # )
                                    break
                        else:
                            continue
                    if task_pending_tmp[0] + task_pending_tmp[1] < self.time:
                        self.end_pending_task(
                            task_pending_tmp[2],
                            task_pending_tmp[3],
                            task_pending_tmp[4],
                        )
                        tmp_pendtsk_list2.remove(task_pending_tmp)

                # once it has been put in task-queue，ends loop
                if tmp_pendtsk_list2:
                    for item in tmp_pendtsk_list2:
                        self.task_pending.put(item)

                if break2:
                    # tmp_allexist_task_arrivetime3 = [i for i, j, k in self.allexist_task_list]
                    # next_uid = [k for i, j, k in self.allexist_task_list if i == min(tmp_allexist_task_arrivetime3)][0]
                    # self.next_uid_idx = next_uid
                    # self.next_time = min(tmp_allexist_task_arrivetime3)
                    next_task_id = []
                    tmp_allexist_task_arrivetime3 = [
                        i for i, j, k in self.allexist_task_list
                    ]
                    next_uid = [
                        k
                        for i, j, k in self.allexist_task_list
                        if i == min(tmp_allexist_task_arrivetime3)
                    ]
                    self.next_uid_idx = next_uid[0]
                    min_task_id = [
                        item.usr_has_tsk
                        for item in self.User
                        if item.uid == next_uid[0]
                    ][0][0]
                    for id in next_uid:
                        task_id = [
                            item.usr_has_tsk for item in self.User if item.uid == id
                        ][0][0]
                        if task_id < min_task_id:
                            min_task_id = task_id
                            self.next_uid_idx = id
                        next_task_id.append(task_id)

                    self.next_time = min(tmp_allexist_task_arrivetime3)
                    break

                if self.task_pending.empty() == True:
                    self.next_uid_idx = None
                    # actually ends
                    break

                self.time += 1
                for idx, usr in enumerate(self.User):
                    self.one_user_loc_update(idx, self.time)

        elif self.task_queue.empty() == True and self.task_pending.empty() == True:
            # actually ends
            self.next_uid_idx = None
            pass

        # get the observation
        obs = self.get_observation()
        # get done
        done = self.is_done()

        total_download_size = max(
            self.tsk_mig_layer_downloadsize.get((pending_tskid, cur_uid), None),
            self.down_size.get((pending_tskid, cur_uid), None),
        )

        reward = reward

        computation_delay = self.tsk_computation_delay.get(
            (pending_tskid, cur_uid), None
        )
        scaling_delay = self.tsk_scaling_delay.get((pending_tskid, cur_uid), None)
        arrange_delay = self.tsk_arrange_delay.get((pending_tskid, cur_uid), None)
        # print("computation_delay:"+ str(computation_delay*1000))
        # print("arrange_delay:"+ str(arrange_delay*1000))
        # print("count_num_on_cloud"+str(count_num_on_cloud))
        # print("------------------------------------------")
        # print("------------------------------------------")
        backhaul_delay = self.tsk_backhaul_delaytime.get((pending_tskid, cur_uid), None)

        migration = self.tsk_mig_dlytime.get((pending_tskid, cur_uid), None)
        total_download_time = max(
            self.tsk_mig_layer_dlytime.get((pending_tskid, cur_uid), None),
            self.tsk_dwnlod_dlytime.get((pending_tskid, cur_uid), None),
        )
        trans = self.usr_up_trans_dlytime1.get((pending_tskid, cur_uid), None)
        waiting_time = self.tsk_waiting_delay.get((pending_tskid, cur_uid), None)

        conatiner_available_node_id = []
        for i, node in enumerate(self.Edge):
            if node.container_number < config.node_max_container_number:
                conatiner_available_node_id.append(node)
        info = conatiner_available_node_id

        details = [
            migration,
            total_download_time,
            trans,
            waiting_time,
            total_download_size,
            computation_delay,
            arrange_delay,
            scaling_delay,
            backhaul_delay,
        ]

        return obs, reward, info, done, details

    def remove_layers(self, action, current_tskid):
        cache_policy = config.layer_remove_policy
        cache_level = config.cache_level
        if cache_policy == "LFU_Size":
            if cache_level == "layer":
                for node_id in range(config.EDGE_NODE_NUM):
                    layer_sorted_frequency = self.lfu_cache_layer[node_id].get_all()
                    if not layer_sorted_frequency:
                        break
                    removed_layer_id = layer_sorted_frequency[0]
                    removed_layer_index = 0
                    node_free_disk_percent = self.get_node_free_disk_percent(node_id)

                    while node_free_disk_percent <= config.lfu_min_free_disk_percent:
                        # if all layers are required, we can not delete anymore and break

                        if removed_layer_index == len(layer_sorted_frequency):
                            break
                        # removed_layer_id = self.lfu_cache_layer[node_id].get()
                        # check if is running, if is in the request
                        # TODO find image id list, check if the corresponding container is running

                        image_list_of_removed_layer = self.Layer[
                            removed_layer_id
                        ].image_list
                        removed_layer_is_running = False
                        for image in image_list_of_removed_layer:
                            image_id = [
                                img.id for img in self.Image if img.name == image
                            ][0]
                            if self.Edge[node_id].container_running_list[image_id] > 0:
                                removed_layer_is_running = True  # Image running
                        #
                        removed_layer_is_requested = (
                            False  # image running，layer cannot be deleted
                        )
                        if node_id == action:
                            removed_layer_name = self.Layer[removed_layer_id].name
                            current_task_requested_image_layer_list = [
                                i.layer_list
                                for i in self.Image
                                if i.name == self.Task[current_tskid].task_image_name[0]
                            ]
                            if (
                                removed_layer_name
                                in current_task_requested_image_layer_list
                            ):
                                removed_layer_is_requested = True
                        if removed_layer_is_running or removed_layer_is_requested:
                            removed_layer_index += 1
                            print("not delete!!!!")
                            if removed_layer_index >= len(layer_sorted_frequency):
                                continue
                            removed_layer_id = layer_sorted_frequency[
                                removed_layer_index
                            ]
                            continue
                        # delete from node layer list
                        print("removed_layer_id is: ", removed_layer_id)
                        self.Edge[node_id].layer_01_list[removed_layer_id] = 0
                        # delete from lfu cache
                        self.lfu_cache_layer[node_id].remove(removed_layer_id)
                        # delete from node ini_lyerid
                        self.Edge[node_id].ini_lyerid.remove(removed_layer_id)
                        # delete from node disk
                        self.Edge[node_id].available_disk += self.Layer[
                            removed_layer_id
                        ].size

                        # ADD delete from node image list
                        removed_image_ids = [
                            i.id
                            for x in self.Layer[removed_layer_id].image_list
                            for i in self.Image
                            if i.name == x
                        ]  # image的名字找id
                        for image_id in removed_image_ids:
                            self.Edge[node_id].image_01_list[image_id] = 0

                        removed_layer_index += 1
                        if removed_layer_index < len(layer_sorted_frequency):
                            removed_layer_id = layer_sorted_frequency[
                                removed_layer_index
                            ]
                        node_free_disk_percent = self.get_node_free_disk_percent(
                            node_id
                        )

    def check_d(self, tsk_pending):
        tsk_offload_edge = tsk_pending[-2]
        if tsk_offload_edge != config.EDGE_NODE_NUM:
            tsk_current_x = [
                itm.location_x
                for ind, itm in enumerate(self.Edge)
                if itm.id == tsk_pending[4]
            ][0]
            tsk_current_y = [
                itm.location_y
                for ind, itm in enumerate(self.Edge)
                if itm.id == tsk_pending[4]
            ][0]
            uid_current_x = [
                itm.user_locationx
                for ind, itm in enumerate(self.User)
                if itm.uid == tsk_pending[3]
            ][0]
            uid_current_y = [
                itm.user_locationy
                for ind, itm in enumerate(self.User)
                if itm.uid == tsk_pending[3]
            ][0]
            if uid_current_x == np.inf or uid_current_y == np.inf:
                return True
            d = delay.bts_distance(
                tsk_current_x, tsk_current_y, uid_current_x, uid_current_y
            )

            if d > config.TRANSMISSION_RANGE:
                # mig
                return True
            else:
                # no mig
                return False
        else:
            # on cloud no mig
            return False

    def end_pending_task(self, tskid, uid, edgeid):  
        # 释放资源  
        usr_belong_edgeid2 = edgeid  
        tskid2 = tskid  
        requested_imge_id2 = [ii.id for ii in self.Image if ii.name == self.Task[tskid2].task_image_name[0]][0]  
        
        if usr_belong_edgeid2 != config.EDGE_NODE_NUM:  
            self.remove_layers(usr_belong_edgeid2, tskid2)  
            
            # 释放资源  
            self.Edge[usr_belong_edgeid2].available_cpu += self.Task[tskid2].task_cpu  
            self.Edge[usr_belong_edgeid2].available_mem += self.Task[tskid2].task_mem  
            self.Edge[usr_belong_edgeid2].container_running_list[requested_imge_id2] -= 1  
            self.Edge[usr_belong_edgeid2].image_01_list[requested_imge_id2] = 0  
            self.Edge[usr_belong_edgeid2].container_number -= 1  
            self.edge_task_queue[usr_belong_edgeid2][tskid2] = 0  
        
        self.tp_edgcount[edgeid] -= 1  
        
        # 更新用户状态，而不是删除  
        indix = [ind for ind, itm in enumerate(self.User) if itm.uid == uid][0]  
        
        # 重置用户任务状态，而不是删除用户  
        self.User[indix].usr_has_tsk = []  
        
        # 从user_group中移除，但不删除用户  
        for usr_lst in self.user_group:  
            if uid in usr_lst:  
                usr_lst.remove(uid)  

    def get_node_free_disk_percent(self, node_id):
        node = self.Edge[node_id]
        layer_size = 0
        for index, layer_status in enumerate(node.layer_01_list):
            if layer_status == -1:
                layer_size += int(self.Layer[index].size)
        return 1 - layer_size / node.disk

    def thruUsrGroup_find_usrCurEdg(self, pending_tskid):
        prei = None
        # for nodeid,item in enumerate(self.user_group):
        #     if int(uid) in item:
        #         prei = nodeid
        #         break
        node_id = self.Task[pending_tskid].assigned_node
        if node_id is not None:
            prei = node_id

        return prei

    def generate_user_layer_list2(self, tmp_sample_tsk_id):  # one usr
        layer_list = []
        for task in self.Task:
            if task.task_id == tmp_sample_tsk_id:
                img_namelst = task.task_image_name
                for img_name in img_namelst:
                    layer_list += [
                        item.layer_list for item in self.Image if img_name == item.name
                    ][0]
            else:
                pass

        layer_list = list(set(layer_list))
        # return layer_list
        new = []
        for lyer_name in layer_list:
            for item in self.Layer:
                if lyer_name == item.name:
                    new.append(item.id)
        return new

    def one_user_loc_update(self, idx, time):
        arrive_time = self.User[idx].tsk_arrive_time_start
        if time <= arrive_time:
            thr = time
        else:
            thr = time - arrive_time
        if thr < len(self.mob[idx][1]):
            self.User[idx].user_locationx = self.mob[idx][1][thr][0]
            self.User[idx].user_locationy = self.mob[idx][1][thr][1]
        else:
            # out range put on cloud
            self.User[idx].user_locationx = np.inf
            self.User[idx].user_locationy = np.inf

    def pool(self, uid, uloc_dict1, eloc_dict2):
        # distance
        usr_choice_pool = np.zeros((config.EDGE_NODE_NUM + 1))

        # 如果uid为-1，只返回云节点
        if uid == -1:
            return [config.EDGE_NODE_NUM]

        ux = [j for i, j, k in uloc_dict1 if i == uid][0]
        uy = [k for i, j, k in uloc_dict1 if i == uid][0]
        can = []
        for oneloc in eloc_dict2:
            d = delay.bts_distance(ux, uy, oneloc[1], oneloc[2])
            # print("distance is: ",d)
            if d < config.TRANSMISSION_RANGE:
                usr_choice_pool[oneloc[0]] = 1
        for idx, item in enumerate(usr_choice_pool):
            if item == 1:
                can.append(idx)
        return can

    def check_scaling(self, node):
        user_list = self.user_group[node.id]
        min_cpu = 10000
        task_id = -1
        for i in user_list:
            temp_task = [
                task.task_id for task in self.Task if task.task_belong_userid == i
            ][0]
            if self.Task[temp_task].cpu < min_cpu:
                min_cpu = self.Task[temp_task].cpu
                task_id = temp_task
        total_operation_time = self.tsk_operation_time.get(
            (self.Task[task_id].task_id, self.Task[task_id].task_belong_userid)
        )

    def workload(self, usr_belong_edgeid):
        tmp_pendtsk_list = []
        current_workload = 0

        while self.task_pending.empty() == False:
            task_pending_tmp = self.task_pending.get()
            tmp_pendtsk_list.append(task_pending_tmp)
            node = self.Task[task_pending_tmp[2]].assigned_node
            if usr_belong_edgeid == node:
                task_siz = [
                    i.size
                    for i in self.Image
                    if i.name == self.Task[task_pending_tmp[2]].task_image_name[0]
                ][0]
                current_workload += (
                    task_siz * 1024 * 1024 * 8 * config.task_process_density
                )  # Mb*1024*1024 = b    b* 1000 circle/bit = circle

        if tmp_pendtsk_list:
            for item in tmp_pendtsk_list:
                self.task_pending.put(item)

        return current_workload


if __name__ == "__main__":
    # env = Env()
    # env.reset()
    # obs, r, info, done, details = env.step(env.Edge[2].id)
    # env.get_observation()
    # # node_capacity = [node.cpu for node in env.Edge]
    # print(obs['next_task'][0][0])
    # print("trajectory is: ",str(trajectory))
    # print(node_capacity)
    pass
